#!/usr/bin/env python3
# -*- coding:utf-8 -*-
# @Time: 2019/5/31  17:07
# @Author: 余浪人
# @Email:yulangren520@Gmail.com

# https://www.cnblogs.com/mafeng/p/5651323.html  架构图
from spider.crawler import download_get, queue
from lxml import etree
import re, datetime


def xpath_analysis_list(list_page_url: str, doc_cls: int):
    response = download_get(list_page_url)
    html = etree.HTML(response.text)
    list_li = html.xpath('//ul[@class="articleCon"]/li')
    if len(list_li):
        for li in list_li:
            try:
                detail_url = li.xpath('./a[@class="more"]/@href')[0]  # 解析出 详细页面URl
            except:
                continue
            # 将 详细页面URL添加值下载队列
            queue.put({"detail_url": detail_url, "doc_cls": doc_cls})
    return "空"


def re_analysis_list(list_page_url: str, doc_cls: int):
    response = download_get(list_page_url).text
    List = re.search('<div class="con-left">([\s\S]*?)<div class="con-right">', response, re.S).group(1)  # 列表区域
    if List:
        urlList = re.findall('<h3><a href="(.*)" target="_blank">', List)
        if urlList:
            for detail_url in urlList:
                queue.put({"detail_url": detail_url, "doc_cls": doc_cls})
            return urlList
    return "空"


if __name__ == '__main__':
    list_url = 'http://www.zzlzl.com/sort/1'
    res = re_analysis_list(list_url, 1)
    print(res)